#!/bin/bash


MODEL_VERSION=qwen2-7b-chat
PROMPT_VERSION=qwen

IMG_SIZE=448

PROJ_TYPE=cabstractorTempoAfter_d8
VE_TYPE=siglip
VIDEO_FRAME=12

# Adjust for temporal, now the output is 64 tokens
# MVBench最高54.28分，已经非常不错了，是否可以更高？看起来我们的stage1.5策略是有效的

# d4已经超过很多模型了，77.2吊打bunny，暂时不刷榜了，开始做视频的pretrain

# 先试一下cabstractor是否比perceiver更有效，先把token加到最大。

#                 ./data/sft/share-captioner_coco_lcs_sam_1246k_1107.json \

#                 ./data/sft/llavar_pretrain.json \

#                 ./data/sft/mtwi_ocr_20k.json \
#                 ./data/sft/llavar_pretrain.json \
#     --learning_rate 0.6e-4 \

export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6

# ./data/sft/textmonkey_pretrain.json \
#                 ./data/sft/llava_recap_558k.json \


# deepspeed train_mem.py \
#     --deepspeed ./scripts/zero2.json \
#     --model_name_or_path ./checkpoints/$MODEL_VERSION \
#     --version $PROMPT_VERSION \
#     --data_path ./data/sft/sharegpt4v_mini_pretrain.json \
#                 ./data/sft/llava_recap_cc3m_mini.json \
#     --image_folder ./data/images_all \
#     --vision_tower ./checkpoints/siglip-so400m-patch14-384 \
#     --mm_projector_type $PROJ_TYPE \
#     --new_img_size $IMG_SIZE \
#     --tune_mm_mlp_adapter True \
#     --mm_vision_select_layer -2 \
#     --mm_use_im_start_end False \
#     --mm_use_im_patch_token False \
#     --bf16 True \
#     --output_dir ./checkpoints/llava-$MODEL_VERSION-pretrain-$VE_TYPE-$PROJ_TYPE-$IMG_SIZE \
#     --num_train_epochs 1 \
#     --per_device_train_batch_size 11 \
#     --per_device_eval_batch_size 4 \
#     --gradient_accumulation_steps 2 \
#     --evaluation_strategy "no" \
#     --save_strategy "steps" \
#     --save_steps 4000 \
#     --save_total_limit 1 \
#     --learning_rate 1e-4 \
#     --weight_decay 0. \
#     --warmup_ratio 0.03 \
#     --lr_scheduler_type "cosine" \
#     --logging_steps 1 \
#     --tf32 True \
#     --fp16 False \
#     --model_max_length 2048 \
#     --gradient_checkpointing True \
#     --dataloader_num_workers 4 \
#     --lazy_preprocess True


# 想办法增加一个1.5 stage？


# Perceiver的版本，用A100 zero2 batchsize 2 训不起来，大概率是VE太大
# 只能用zero3

# 感觉Bunny里面的projectlr很小，proj和ve的学习速率一直，LLM的学习速率不知道多大
# 我们再Resampler里面，Proj的学习速率几乎是致命的，调不好就蹦。

#                 ./data/sft/multi_llava_665k.json \


# ssv2, clever 这两个数据集在MVBench上得分提升具有一定的作用。
# lora_r 128, lora_alpha 256 no dropout weight decay 0 感觉有点过拟合
# 增加一点dropout和weightdecay看看, lora_r 64 lora_alpha 16 dropout=0.05 adam_beta2=0.95
# 如果继续调小rank？

# deepspeed train_mem.py \
#     --deepspeed ./scripts/zero2.json \
#     --lora_enable True --lora_r 64 --lora_alpha 16 --lora_dropout 0.05 --mm_projector_lr 1e-5 \
#     --mm_vision_tower_lr 0.3e-5 \
#     --model_name_or_path ./checkpoints/$MODEL_VERSION \
#     --version $PROMPT_VERSION \
#     --data_path ./data/sft/llava_recap_cc3m_mini.json \
#                 ./data/sft/textmonkey_pretrain.json \
#     --image_folder ./data/images_all \
#     --vision_tower ./checkpoints/siglip-so400m-patch14-384 \
#     --video_frames_num $VIDEO_FRAME \
#     --new_img_size $IMG_SIZE \
#     --unfreeze_mm_vision_tower True \
#     --mm_projector_type $PROJ_TYPE \
#     --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain-$VE_TYPE-$PROJ_TYPE-$IMG_SIZE/mm_projector.bin \
#     --mm_vision_select_layer -2 \
#     --mm_use_im_start_end False \
#     --mm_use_im_patch_token False \
#     --image_aspect_ratio pad \
#     --group_by_modality_length True \
#     --bf16 True \
#     --output_dir ./checkpoints/llava-$MODEL_VERSION-$VE_TYPE-$PROJ_TYPE-vit-lora-$IMG_SIZE \
#     --num_train_epochs 1 \
#     --per_device_train_batch_size 5 \
#     --per_device_eval_batch_size 4 \
#     --gradient_accumulation_steps 4 \
#     --evaluation_strategy "no" \
#     --save_strategy "steps" \
#     --save_steps 1000 \
#     --save_total_limit 1 \
#     --learning_rate 1e-5 \
#     --weight_decay 0.01 \
#     --warmup_ratio 0.03 \
#     --adam_beta2 0.95 \
#     --lr_scheduler_type "cosine" \
#     --logging_steps 1 \
#     --tf32 True \
#     --model_max_length 2048 \
#     --gradient_checkpointing True \
#     --dataloader_num_workers 4 \
#     --lazy_preprocess True 


# Stage 2.0, continue training a new epoch will less data

#                 
                # ./data/sft/reasoning_next_qa.json \
                # ./data/sft/ssv2.json \


# 感觉stage2之后模型更聪明了？

                # ./data/sft/sharegpt4o_video.jsonl \

    # --pretrain_stage_1_5 ./checkpoints/llava-$MODEL_VERSION-$VE_TYPE-$PROJ_TYPE-vit-lora-$IMG_SIZE \

                # ./data/sft/sharegpt4v_mix665k_cap23k_coco-ap9k_lcs3k_sam9k_div2k.json \
                # ./data/sft/multi_llava_665k.json \


deepspeed train_mem.py \
    --deepspeed ./scripts/zero2.json \
    --lora_enable True --lora_r 512 --lora_alpha 256 --lora_dropout 0.05 --mm_projector_lr 1e-5 \
    --mm_vision_tower_lr 1e-5 \
    --model_name_or_path ./checkpoints/$MODEL_VERSION \
    --version $PROMPT_VERSION \
    --data_path ./data/sft/bunny_695k.json \
                ./data/sft/multi_lrv_multi.json \
                ./data/sft/multi_spot_diff.json \
                ./data/sft/videochat2_video.json \
                ./data/sft/ego_video.json \
                ./data/sft/vcg-plus_112K.json \
                ./data/sft/k710.json \
                ./data/sft/reasoning_clevrer_mc.json \
                ./data/sft/reasoning_clevrer_qa.json \
                ./data/sft/reasoning_next_qa.json \
                ./data/sft/ssv2.json \
    --image_folder ./data/images_all \
    --vision_tower ./checkpoints/siglip-so400m-patch14-384 \
    --video_frames_num $VIDEO_FRAME \
    --new_img_size $IMG_SIZE \
    --unfreeze_mm_vision_tower True \
    --mm_projector_type $PROJ_TYPE \
    --mm_vision_select_layer -2 \
    --mm_use_im_start_end False \
    --mm_use_im_patch_token False \
    --image_aspect_ratio pad \
    --group_by_modality_length True \
    --bf16 True \
    --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain-$VE_TYPE-$PROJ_TYPE-$IMG_SIZE/mm_projector.bin \
    --output_dir ./checkpoints/llava-$MODEL_VERSION-$VE_TYPE-$PROJ_TYPE-vit-lora-$IMG_SIZE \
    --num_train_epochs 1 \
    --per_device_train_batch_size 5 \
    --per_device_eval_batch_size 4 \
    --gradient_accumulation_steps 4 \
    --evaluation_strategy "no" \
    --save_strategy "steps" \
    --save_steps 1000 \
    --save_total_limit 1 \
    --learning_rate 2e-5 \
    --weight_decay 0. \
    --warmup_ratio 0.03 \
    --lr_scheduler_type "cosine" \
    --logging_steps 1 \
    --tf32 True \
    --model_max_length 2048 \
    --gradient_checkpointing True \
    --dataloader_num_workers 4 \
    --lazy_preprocess True 
